package com.openness.example.crawler;

import com.openness.crawler.crawler.CrawlConfig;
import com.openness.crawler.crawler.CrawlController;
import com.openness.crawler.fetcher.PageFetcher;
import com.openness.crawler.robotstxt.RobotstxtConfig;
import com.openness.crawler.robotstxt.RobotstxtServer;

/**
 * @author Rain
 * @version 2013-5-20 下午2:57:44
 */

public class MyController {

	public static void main(String[] args) throws Exception {
		CrawlConfig config = new CrawlConfig();

		config.setHbaseMaster("leaderman.rain.centos");

		config.setHbaseZooKeeperQuorum("leaderman.rain.centos");

		config.setHbaseZooKeeperPropertyClientPort("2181");

		config.setHbaseTablePoolSize(5);

		config.setZookeeperSessionTimeout(1000);

		config.setNumberOfCrawlers(5);

		config.setNextUrlSize(100);

		config.setMaxPagesToFetch(100000);

		config.setResumableCrawling(false);

		config.setMaxDepthOfCrawling(2);

		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();

		PageFetcher pageFetcher = new PageFetcher(config);

		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig,
				pageFetcher);

		CrawlController controller = new CrawlController(config, pageFetcher,
				robotstxtServer);

		controller.addSeed("http://www.sina.com.cn/");
		controller.addSeed("http://www.qq.com/");
		controller.addSeed("http://www.sohu.com/");
		controller.addSeed("http://www.163.com/");

		controller.start(MyCrawler.class);
	}

}
